<!DOCTYPE html>












  


<html class="theme-next muse use-motion" lang="zh-CN">
<head><meta name="generator" content="Hexo 3.9.0">
  <meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">








  <script>
  (function(i,s,o,g,r,a,m){i["DaoVoiceObject"]=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;a.charset="utf-8";m.parentNode.insertBefore(a,m)})(window,document,"script",('https:' == document.location.protocol ? 'https:' : 'http:') + "//widget.daovoice.io/widget/0f81ff2f.js","daovoice")
  daovoice('init', {
      app_id: "6db189de"
    });
  daovoice('update');
  </script>









  <link rel="apple-touch-icon" sizes="180x180" href="/images/logo.jpg?v=7.2.0">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/logo.jpg?v=7.2.0">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/logo.jpg?v=7.2.0">







<link rel="stylesheet" href="/css/main.css?v=7.2.0">






<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css?v=4.7.0">


  
  
  <link rel="stylesheet" href="/lib/needsharebutton/needsharebutton.css">







<script id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Muse',
    version: '7.2.0',
    sidebar: {"position":"left","display":"post","offset":12,"onmobile":true},
    back2top: {"enable":true,"sidebar":false,"scrollpercent":true},
    copycode: {"enable":false,"show_result":false,"style":null},
    fancybox: false,
    mediumzoom: false,
    lazyload: false,
    pangu: false,
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    },
    localsearch: {"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},
    search: {
      root: '/',
      path: 'search.xml'
    },
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    translation: {
      copy_button: '复制',
      copy_success: '复制成功',
      copy_failure: '复制失败'
    }
  };
</script>
<script src="//cdn.bootcss.com/pace/1.0.2/pace.min.js"></script>
<link href="//cdn.bootcss.com/pace/1.0.2/themes/pink/pace-theme-flash.css" rel="stylesheet">
  <meta name="description" content="友链:  吴恩达老师的机器学习课程个人笔记： https://github.com/fengdu78/Coursera-ML-AndrewNg-Notes 吴恩达老师的深度学习课程笔记及资源： https://github.com/fengdu78/deeplearning_ai_books github上一个100天学习机器学习的项目： https://github.com/MLEveryd">
<meta name="keywords" content="ML,DL">
<meta property="og:type" content="article">
<meta property="og:title" content="[加密文章]机器学习深度学习笔记">
<meta property="og:url" content="https://swhaledcc.github.io/2019/09/22/机器学习/index.html">
<meta property="og:site_name" content="Dccun&#39;s Blog">
<meta property="og:description" content="友链:  吴恩达老师的机器学习课程个人笔记： https://github.com/fengdu78/Coursera-ML-AndrewNg-Notes 吴恩达老师的深度学习课程笔记及资源： https://github.com/fengdu78/deeplearning_ai_books github上一个100天学习机器学习的项目： https://github.com/MLEveryd">
<meta property="og:locale" content="zh-CN">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-0.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-1.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-2.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-3.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-11.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-12.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-13.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-119.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-120.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-121.png">
<meta property="og:image" content="https://swhaledcc.github.io/images/pasted-116.png">
<meta property="og:updated_time" content="2020-07-24T06:40:23.786Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="[加密文章]机器学习深度学习笔记">
<meta name="twitter:description" content="友链:  吴恩达老师的机器学习课程个人笔记： https://github.com/fengdu78/Coursera-ML-AndrewNg-Notes 吴恩达老师的深度学习课程笔记及资源： https://github.com/fengdu78/deeplearning_ai_books github上一个100天学习机器学习的项目： https://github.com/MLEveryd">
<meta name="twitter:image" content="https://swhaledcc.github.io/images/pasted-0.png">



  <link rel="alternate" href="/atom.xml" title="Dccun's Blog" type="application/atom+xml">



  
  
  <link rel="canonical" href="https://swhaledcc.github.io/2019/09/22/机器学习/">



<script>
    (function(){
        if('199812'){
            if (prompt('请输入查看密码') !== '199812'){
                alert('密码不正确,请询问主编大大！');
                history.back();
            }
        }
    })();
</script>
<script id="page.configurations">
  CONFIG.page = {
    sidebar: "",
  };
</script>

  
  <title>[加密文章]机器学习深度学习笔记 | Dccun's Blog</title>
  












  <noscript>
  <style>
  .use-motion .motion-element,
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-title { opacity: initial; }

  .use-motion .logo,
  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-CN">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>
	<a href="https://github.com/swhaleDCC/swhaleDCC.github.io" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewbox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; left: 0; transform: scale(-1, 1);" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"/><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"/><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"/></svg><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style></a>
    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta">
    

    <div class="custom-logo-site-title">
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">Dccun's Blog</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
    
      
        <p class="site-subtitle">不烦世事，满心欢喜</p>
      
    
    
  </div>

  <div class="site-nav-toggle">
    <button aria-label="切换导航栏">
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>



<nav class="site-nav">
  
    <ul id="menu" class="menu">
      
        
        
        
          
          <li class="menu-item menu-item-home">

    
    
      
    

    

    <a href="/" rel="section"><i class="menu-item-icon fa fa-fw fa-home                          //首页"></i> <br>首页</a>

  </li>
        
        
        
          
          <li class="menu-item menu-item-archives">

    
    
      
    

    

    <a href="/archives/" rel="section"><i class="menu-item-icon fa fa-fw fa-archive          //归档"></i> <br>归档</a>

  </li>
        
        
        
          
          <li class="menu-item menu-item-categories">

    
    
      
    

    

    <a href="/categories/" rel="section"><i class="menu-item-icon fa fa-fw fa-th           //分类"></i> <br>分类</a>

  </li>
        
        
        
          
          <li class="menu-item menu-item-tags">

    
    
      
    

    

    <a href="/tags/" rel="section"><i class="menu-item-icon fa fa-fw fa-tags                     //标签"></i> <br>标签</a>

  </li>
        
        
        
          
          <li class="menu-item menu-item-about">

    
    
      
    

    

    <a href="/about/" rel="section"><i class="menu-item-icon fa fa-fw fa-user                   //关于"></i> <br>关于</a>

  </li>

      
      
        <li class="menu-item menu-item-search">
          <a href="javascript:;" class="popup-trigger">
          
            <i class="menu-item-icon fa fa-search fa-fw"></i> <br>搜索</a>
        </li>
      
    </ul>
  

  
    

  

  
    <div class="site-search">
      
  <div class="popup search-popup local-search-popup">
  <div class="local-search-header clearfix">
    <span class="search-icon">
      <i class="fa fa-search"></i>
    </span>
    <span class="popup-btn-close">
      <i class="fa fa-times-circle"></i>
    </span>
    <div class="local-search-input-wrapper">
      <input autocomplete="off" placeholder="搜索..." spellcheck="false" type="text" id="local-search-input">
    </div>
  </div>
  <div id="local-search-result"></div>
</div>



    </div>
  
</nav>



</div>
    </header>

    


    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
      

  
  
  

  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="https://swhaledcc.github.io/2019/09/22/机器学习/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="Dccun">
      <meta itemprop="description" content="你是我心上百褶，是人间惊鸿客">
      <meta itemprop="image" content="/images/dcc.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Dccun's Blog">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">[加密文章]机器学习深度学习笔记

              
            
          </h1>
        

        <div class="post-meta">
        	


          
          
          

          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              

              
                
              

              <time title="创建时间：2019-09-22 20:12:00" itemprop="dateCreated datePublished" datetime="2019-09-22T20:12:00+08:00">2019-09-22</time>
            </span>
          

          

          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/categories/深度学习/" itemprop="url" rel="index"><span itemprop="name">深度学习</span></a></span>

                
                
              
            </span>
          

          
          

          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
                 阅读次数： 
                <span class="busuanzi-value" id="busuanzi_value_page_pv"></span>
              </span>
            </span>
          

          

          <br>
          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <p><img src="/images/pasted-0.png" alt="upload successful"></p>
<blockquote>
<p>友链:</p>
<ul>
<li>吴恩达老师的机器学习课程个人笔记： <a href="https://github.com/fengdu78/Coursera-ML-AndrewNg-Notes" target="_blank" rel="noopener">https://github.com/fengdu78/Coursera-ML-AndrewNg-Notes</a></li>
<li>吴恩达老师的深度学习课程笔记及资源： <a href="https://github.com/fengdu78/deeplearning_ai_books" target="_blank" rel="noopener">https://github.com/fengdu78/deeplearning_ai_books</a></li>
<li>github上一个100天学习机器学习的项目：<ul>
<li><a href="https://github.com/MLEveryday/100-Days-Of-ML-Code" target="_blank" rel="noopener">https://github.com/MLEveryday/100-Days-Of-ML-Code</a></li>
<li><a href="https://github.com/Avik-Jain/100-Days-Of-ML-Code" target="_blank" rel="noopener">https://github.com/Avik-Jain/100-Days-Of-ML-Code</a></li>
</ul>
</li>
</ul>
</blockquote>
<a id="more"></a>

<h1 id="深度学习框架"><a href="#深度学习框架" class="headerlink" title="深度学习框架"></a>深度学习框架</h1><blockquote>
<p>全世界最为流行的深度学习框架有PaddlePaddle、Tensorflow、Caffe、Theano、MXNet、Torch和PyTorch。<br>conda是一个流行的Python包管理软件。<br><img src="/images/pasted-1.png" alt="upload successful"></p>
</blockquote>
<h1 id="特征缩放"><a href="#特征缩放" class="headerlink" title="特征缩放"></a>特征缩放</h1><blockquote>
<p>特征缩放(feature scaling)<br>梯度下降算法中，在有多个特征的情况下，如果能确保这些不同的特征都处在一个相近的范围，这样梯度下降法就能更快地收敛。</p>
</blockquote>
<h1 id="MNIST-Dataset"><a href="#MNIST-Dataset" class="headerlink" title="MNIST Dataset"></a>MNIST Dataset</h1><blockquote>
<p><a href="https://github.com/aymericdamien/TensorFlow-Examples/blob/master/notebooks/0_Prerequisite/mnist_dataset_intro.ipynb" target="_blank" rel="noopener">MNIST Dataset Introduction</a><br>mnist数据集里的每张图片大小为28 * 28像素，可以用28*28的大小的数组来表示一张图片。<br>标签用大小为10的数组来表示，这种编码我们称之为One hot（独热编码）。</p>
</blockquote>
<h1 id="One-hot编码"><a href="#One-hot编码" class="headerlink" title="One-hot编码"></a>One-hot编码</h1><blockquote>
<p>One-hot编码（独热编码）<br>独热编码使用N位代表N种状态，任意时候只有其中一位有效。<br>采用独热编码的例子<br>性别:<br>[0, 1]代表女，[1, 0]代表男<br>数字0-9:<br>[0,0,0,0,0,0,0,0,0,1]代表9，[0,1,0,0,0,0,0,0,0,0]代表1</p>
</blockquote>
<h1 id="损失函数（loss-function）"><a href="#损失函数（loss-function）" class="headerlink" title="损失函数（loss function）"></a>损失函数（loss function）</h1><blockquote>
<p>常见的损失函数定义:</p>
<ul>
<li>差的平方和 sum((y - label)^2)</li>
<li><a href="https://zhuanlan.zhihu.com/p/35709485" target="_blank" rel="noopener">交叉熵</a> -sum(label * log(y))(交叉熵只关注独热编码中有效位的损失)</li>
</ul>
</blockquote>
<h1 id="梯度下降"><a href="#梯度下降" class="headerlink" title="梯度下降"></a>梯度下降</h1><p><img src="/images/pasted-2.png" alt="upload successful"><br>learning_rate:学习速率</p>
<h1 id="softmax激活函数"><a href="#softmax激活函数" class="headerlink" title="softmax激活函数"></a>softmax激活函数</h1><blockquote>
<p>作用:一是放大效果，二是梯度下降时需要一个可导的函数。</p>
</blockquote>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">def softmax(x):</span><br><span class="line">    import numpy as np</span><br><span class="line">    return np.exp(x) / np.sum(np.exp(x), axis=0)</span><br><span class="line">softmax([4, 5, 10])</span><br></pre></td></tr></table></figure>

<h1 id="tensorflow-playground"><a href="#tensorflow-playground" class="headerlink" title="tensorflow playground"></a>tensorflow playground</h1><p>附上一张我试着训练出来的loss最小的螺旋神经网络<br><img src="/images/pasted-3.png" alt="upload successful"></p>
<h1 id="混淆矩阵"><a href="#混淆矩阵" class="headerlink" title="混淆矩阵"></a>混淆矩阵</h1><p>混淆矩阵也称误差矩阵，是表示精度评价的一种标准格式，用n行n列的矩阵形式来表示。在人工智能中，混淆矩阵（confusion matrix）是可视化工具，特别用于监督学习，在无监督学习一般叫做匹配矩阵。</p>
<p><img src="/images/pasted-11.png" alt="upload successful"></p>
<p><img src="/images/pasted-12.png" alt="upload successful"></p>
<p><img src="/images/pasted-13.png" alt="upload successful"></p>
<h1 id="KL散度、JS散度、Wasserstein距离"><a href="#KL散度、JS散度、Wasserstein距离" class="headerlink" title="KL散度、JS散度、Wasserstein距离"></a><a href="https://www.cnblogs.com/jiangxinyang/p/10563113.html" target="_blank" rel="noopener">KL散度、JS散度、Wasserstein距离</a></h1><p><img src="/images/pasted-119.png" alt="upload successful"></p>
<p><img src="/images/pasted-120.png" alt="upload successful"></p>
<p><img src="/images/pasted-121.png" alt="upload successful"></p>
<h1 id="梯度下降法（BGD-amp-SGD-amp-Mini-batch-SGD）"><a href="#梯度下降法（BGD-amp-SGD-amp-Mini-batch-SGD）" class="headerlink" title="梯度下降法（BGD &amp; SGD &amp; Mini-batch SGD）"></a><a href="https://www.cnblogs.com/lvdongjie/p/11318008.html" target="_blank" rel="noopener">梯度下降法（BGD &amp; SGD &amp; Mini-batch SGD）</a></h1><p><code>梯度下降法（Gradient Descent）</code></p>
<p>优化思想：用当前位置的负梯度方向作为搜索方向，亦即为当前位置下降最快的方向。越接近目标值时，步长越小，下降越慢，梯度下降不一定能找到全局最优解，可能寻找到的是局部最优解。（当损失函数是凸函数时，梯度下降得到的解一定是全局最优解，因为凸函数的极小值即为最小值）</p>
<ul>
<li>批量梯度下降法（Batch Gradient Descent，BGD）：每次是对整个训练集进行梯度下降。</li>
<li>随机梯度下降法（Stochastic Gradient Descent，SGD）：每次只对一个样本进行梯度下降。</li>
<li>小批量梯度下降法（Mini-batch Gradient Desent，也称Mini-batch SGD）：每次处理样本的个数在上面二者之间，把整个大的训练集划分为若干个小的训练集，在处理完整个训练集之前，先让梯度下降法处理一部分数据，那么算法就会相对快一些。当数据集很大时，使用Mini-batch SGD更新参数更快，有利于更鲁棒地收敛，避免局部最优。</li>
</ul>
<p><code>选择Mini-batch SGD的参数 batch size</code></p>
<p>不难看出Mini-batch SGD的 batch 大小，也是一个影响着算法效率的参数。</p>
<p>如果训练集较小，一般小于2000的，就直接使用BGD。</p>
<p>一般Mini-batch SGD的大小在 64 到 512 之间，选择 2 的 n 次幂会运行得相对快一些。注意这个值设为 2 的 n 次幂，是为了符合cpu gpu的内存要求，如果不符合的话，不管用什么算法表现都会很糟糕。</p>
<p><a href="https://www.zhihu.com/question/32673260/answer/71137399" target="_blank" rel="noopener">深度学习中的batch</a></p>
<p>Batch_Size：批尺寸</p>
<p><img src="/images/pasted-116.png" alt="upload successful"></p>
<h1 id="Normalization模型"><a href="#Normalization模型" class="headerlink" title="Normalization模型"></a><a href="https://blog.csdn.net/malefactor/article/details/82154224" target="_blank" rel="noopener">Normalization模型</a></h1><p>　　在机器学习领域有个很重要的假设：<code>IID独立同分布假设</code>，也就是假设训练数据和测试数据是满足相同分布的，这是通过训练数据获得的模型能够在测试集上获得好的效果的一个基本保障。在深度学习网络中，后一层的输入是受前一层的影响的，而为了方便训练网络，我们一般都是采用Mini-Batch SGD来训练网络的（Mini-Batch SGD的两个优点是：梯度更新方向更准确和并行计算速度快）。</p>
<p>　　我们知道在神经网络训练开始前，都要对输入数据做一个归一化处理，那么具体为什么需要归一化呢？<code>归一化后有什么好处呢</code>？原因在于神经网络学习过程本质就是为了学习数据分布，一旦训练数据与测试数据的分布不同，那么网络的泛化能力也大大降低；另外一方面，一旦每批训练数据的分布各不相同(batch 梯度下降)，那么网络就要在每次迭代都去学习适应不同的分布，这样将会大大降低网络的训练速度，这也正是为什么我们需要对数据都要做一个归一化预处理的原因。</p>
<p>　　对于深度网络的训练是一个复杂的过程，只要网络的前面几层发生微小的改变，那么后面几层就会被累积放大下去。一旦网络某一层的输入数据的分布发生改变，那么这一层网络就需要去适应学习这个新的数据分布，所以如果训练过程中，训练数据的分布一直在发生变化，那么将会影响网络的训练速度。</p>
<p>　　除了输入层的数据外(因为输入层数据，我们已经人为的为每个样本归一化)，后面网络每一层的输入数据分布是一直在发生变化的，因为在训练的时候，前面层训练参数的更新将导致后面层输入数据分布的变化。以网络第二层为例：网络的第二层输入，是由第一层的参数和input计算得到的，而第一层的参数在整个训练过程中一直在变化，因此必然会引起后面每一层输入数据分布的改变。</p>
<p>　　我们把网络中间层在训练过程中，数据分布的改变称之为：“Internal  Covariate Shift”。Internal指的是深层网络的隐层，是发生在网络内部的事情，而不是covariate shift问题只发生在输入层。Batch Normalization就是来解决该问题的。Batch Normalization的基本思想就是能不能让每个隐层节点的激活输入分布固定下来，从而避免Internal Covariate Shift的问题。</p>
<p>　　BN(Batch Normalization)属于网络的一层。BN的基本思想其实相当直观：因为深层神经网络在做非线性变换前的激活输入值（就是那个x=WU+B，U是输入）随着网络深度加深或者在训练过程中，其分布逐渐发生偏移或者变动，之所以训练收敛慢，一般是整体分布逐渐往非线性函数的取值区间的上下限两端靠近（对于Sigmoid函数来说，意味着激活输入值WU+B是大的负值或正值），所以这导致后向传播时低层神经网络的梯度消失，这是训练深层神经网络收敛越来越慢的本质原因，而BN就是通过一定的规范化手段，<code>把每层神经网络任意神经元输入值的分布强行拉回到均值为0方差为1的标准正态分布，使得激活输入值落在非线性函数对输入比较敏感的区域，这样输入的小变化就会导致损失函数较大的变化，意思是这样让梯度变大，避免梯度消失问题产生，而且梯度变大意味着学习收敛速度快，能大大加快训练速度。</code></p>
<h1 id="理解dropout"><a href="#理解dropout" class="headerlink" title="理解dropout"></a>理解dropout</h1><p>参考：<a href="http://blog.csdn.net/stdcoutzyx/article/details/49022443" target="_blank" rel="noopener">http://blog.csdn.net/stdcoutzyx/article/details/49022443</a></p>
<blockquote>
<p>dropout是指在深度学习网络的训练过程中，对于神经网络单元，按照一定的概率将其暂时从网络中丢弃。注意是暂时，对于随机梯度下降来说，由于是随机丢弃，故而每一个mini-batch都在训练不同的网络。</p>
</blockquote>
<p>dropout率的选择：</p>
<ul>
<li>经过交叉验证，隐含节点dropout率等于0.5的时候效果最好，原因是0.5的时候dropout随机生成的网络结构最多。</li>
<li>dropout也可以被用作一种添加噪声的方法，直接对input进行操作。输入层设为更接近1的数。使得输入变化不会太大（0.8）</li>
</ul>

      
    </div>

    

    
      
    

    
    
    
    <div>
  
    <div>
    
        <div style="text-align:center;color: #ccc;font-size:14px;">---------------------------- 本 文 结 束  <i class="fa fa-paw"></i>  感 谢 阅 读 ----------------------------</div>
    
</div>
  
</div>

    

    
      
    

    
      <div>
        <div id="reward-container">
  <div>欢迎打赏~</div>
  <button id="reward-button" disable="enable" onclick="var qr = document.getElementById(&quot;qr&quot;); qr.style.display = (qr.style.display === 'none') ? 'block' : 'none';">
    打赏
  </button>
  <div id="qr" style="display: none;">

    
      
      
        
      
      <div style="display: inline-block">
        <img src="/images/wechat.png" alt="Dccun 微信支付">
        <p>微信支付</p>
      </div>
    

  </div>
</div>

      </div>
    

    

    <footer class="post-footer">
      
        
          
        
        <div class="post-tags">
          
            <a href="/tags/ML/" rel="tag"><i class="fa fa-tag"></i> ML</a>
          
            <a href="/tags/DL/" rel="tag"><i class="fa fa-tag"></i> DL</a>
          
        </div>
      

      
      
        <div class="post-widgets">
        

        

        
          
          <div class="social_share">
            
            
              <div id="needsharebutton-postbottom">
                <span class="btn">
                  <i class="fa fa-share-alt" aria-hidden="true"></i>
                </span>
              </div>
            
          </div>
        
        </div>
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2019/08/07/2019电子竞赛国赛/" rel="next" title="2019电子竞赛准备和比赛过程记录">
                <i class="fa fa-chevron-left"></i> 2019电子竞赛准备和比赛过程记录
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2019/09/30/ubantu入门/" rel="prev" title="ubuntu双系统配置记录">
                ubuntu双系统配置记录 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>

  </div>


          </div>
          
  
    
    
  <div class="comments" id="comments">
    <div id="lv-container" data-id="city" data-uid="MTAyMC80NjM5MC8yMjkwMQ=="></div>
  </div>
  
  



        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-overview">

          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  
    <img class="site-author-image" itemprop="image" src="/images/dcc.jpg" alt="Dccun">
  
  <p class="site-author-name" itemprop="name">Dccun</p>
  <div class="site-description motion-element" itemprop="description">你是我心上百褶，是人间惊鸿客</div>
</div>


  <nav class="site-state motion-element">
    
      <div class="site-state-item site-state-posts">
        
          <a href="/archives/">
        
          <span class="site-state-item-count">31</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
    

    
      
      
      <div class="site-state-item site-state-categories">
        
          
            <a href="/categories/">
          
        
        
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
        <span class="site-state-item-count">11</span>
        <span class="site-state-item-name">分类</span>
        </a>
      </div>
    

    
      
      
      <div class="site-state-item site-state-tags">
        
          
            <a href="/tags/">
          
        
        
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
        <span class="site-state-item-count">27</span>
        <span class="site-state-item-name">标签</span>
        </a>
      </div>
    
  </nav>



  <div class="feed-link motion-element">
    <a href="/atom.xml" rel="alternate">
      <i class="fa fa-rss"></i>RSS
    </a>
  </div>





  <div class="links-of-author motion-element">
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://github.com/swhaleDCC" title="GitHub &rarr; https://github.com/swhaleDCC" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i>GitHub</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://www.weibo.com/5986829280/profile?topnav=1&wvr=6" title="weibo &rarr; https://www.weibo.com/5986829280/profile?topnav=1&wvr=6" rel="noopener" target="_blank"><i class="fa fa-fw fa-weibo"></i>weibo</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://blog.csdn.net/qq_40631927" title="csdn &rarr; https://blog.csdn.net/qq_40631927" rel="noopener" target="_blank"><i class="fa fa-fw fa-crosshairs"></i>csdn</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://www.jianshu.com/u/e048aac8c8a0" title="简书 &rarr; https://www.jianshu.com/u/e048aac8c8a0" rel="noopener" target="_blank"><i class="fa fa-fw fa-heartbeat"></i>简书</a>
      </span>
    
  </div>






  <div class="links-of-blogroll motion-element links-of-blogroll-inline">
    <div class="links-of-blogroll-title">
      <i class="fa  fa-fw fa-link"></i>
      友情链接
    </div>
    <ul class="links-of-blogroll-list">
      
        <li class="links-of-blogroll-item">
          <a href="https://www.zhihu.com/people/zhe-yan-56-43/activities" title="https://www.zhihu.com/people/zhe-yan-56-43/activities" rel="noopener" target="_blank">知乎</a>
        </li>
      
        <li class="links-of-blogroll-item">
          <a href="https://zhuanlan.zhihu.com/p/32957389" title="https://zhuanlan.zhihu.com/p/32957389" rel="noopener" target="_blank">github+hexo搭建个人博客</a>
        </li>
      
        <li class="links-of-blogroll-item">
          <a href="https://www.jianshu.com/p/1f8107a8778c" title="https://www.jianshu.com/p/1f8107a8778c" rel="noopener" target="_blank">next主题优化</a>
        </li>
      
        <li class="links-of-blogroll-item">
          <a href="https://lab.github.com/" title="https://lab.github.com/" rel="noopener" target="_blank">GitHub Learning Lab</a>
        </li>
      
    </ul>
  </div>


          
          
        </div>
      </div>

      
      <!--noindex-->
        <div class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
            
            
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#深度学习框架"><span class="nav-text">深度学习框架</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#特征缩放"><span class="nav-text">特征缩放</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#MNIST-Dataset"><span class="nav-text">MNIST Dataset</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#One-hot编码"><span class="nav-text">One-hot编码</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#损失函数（loss-function）"><span class="nav-text">损失函数（loss function）</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#梯度下降"><span class="nav-text">梯度下降</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#softmax激活函数"><span class="nav-text">softmax激活函数</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#tensorflow-playground"><span class="nav-text">tensorflow playground</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#混淆矩阵"><span class="nav-text">混淆矩阵</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#KL散度、JS散度、Wasserstein距离"><span class="nav-text">KL散度、JS散度、Wasserstein距离</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#梯度下降法（BGD-amp-SGD-amp-Mini-batch-SGD）"><span class="nav-text">梯度下降法（BGD &amp; SGD &amp; Mini-batch SGD）</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Normalization模型"><span class="nav-text">Normalization模型</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#理解dropout"><span class="nav-text">理解dropout</span></a></li></ol></div>
            

          </div>
        </div>
      <!--/noindex-->
      

      

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; 2019 – <span itemprop="copyrightYear">2022</span>
  <span class="with-love" id="animate">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Dccun</span>

  

  
</div>
<!--

  <div class="powered-by">由 <a href="https://hexo.io" class="theme-link" rel="noopener" target="_blank">Hexo</a> 强力驱动 v3.9.0</div>



  <span class="post-meta-divider">|</span>



  <div class="theme-info">主题 – <a href="https://theme-next.org" class="theme-link" rel="noopener" target="_blank">NexT.Muse</a> v7.2.0</div>


-->

<div class="theme-info">
  <div class="powered-by"></div>
  <span class="post-count">博客全站共32.9k字</span>
</div>
        
<div class="busuanzi-count">
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>

  
    <span class="post-meta-item-icon">
      <i class="fa fa-user"></i>
    </span>
    <span class="site-uv" title="总访客量">
      <span class="busuanzi-value" id="busuanzi_value_site_uv"></span>
    </span>
  

  
    <span class="post-meta-divider">|</span>
  

  
    <span class="post-meta-item-icon">
      <i class="fa fa-eye"></i>
    </span>
    <span class="site-pv" title="总访问量">
      <span class="busuanzi-value" id="busuanzi_value_site_pv"></span>
    </span>
  
</div>









        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
          <span id="scrollpercent"><span>0</span>%</span>
        
      </div>
    

    

    
  
  
  <script src="/lib/needsharebutton/needsharebutton.js"></script>
  <script>
    
      pbOptions = {};
      
        pbOptions.iconStyle = "box";
      
        pbOptions.boxForm = "horizontal";
      
        pbOptions.position = "bottomCenter";
      
        pbOptions.networks = "Weibo,Wechat,Douban,QQZone,Twitter,Facebook";
      
      new needShareButton('#needsharebutton-postbottom', pbOptions);
    
    
  </script>


    

  </div>

  

<script>
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>
















  
  



  
    
    
  
  <script color="0,0,255" opacity="0.5" zindex="-1" count="99" src="/lib/canvas-nest/canvas-nest.min.js"></script>







  
  <script src="/lib/jquery/index.js?v=3.4.1"></script>

  
  <script src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>




  <script src="/js/utils.js?v=7.2.0"></script>

  <script src="/js/motion.js?v=7.2.0"></script>



  
  


  <script src="/js/schemes/muse.js?v=7.2.0"></script>



  
  <script src="/js/scrollspy.js?v=7.2.0"></script>
<script src="/js/post-details.js?v=7.2.0"></script>



  <script src="/js/next-boot.js?v=7.2.0"></script>

  

  

  


  













  <script src="/js/local-search.js?v=7.2.0"></script>



















  
    



<script>
  window.livereOptions = {
    refer: '2019/09/22/机器学习/'
  };
  (function(d, s) {
    var j, e = d.getElementsByTagName(s)[0];
    if (typeof LivereTower === 'function') { return; }
    j = d.createElement(s);
    j.src = 'https://cdn-city.livere.com/js/embed.dist.js';
    j.async = true;
    e.parentNode.insertBefore(j, e);
  })(document, 'script');
</script>


  
  <script type="text/javascript" src="/js/src/love.js"></script>
  <script type="text/javascript" src="/js/src/particle.js"></script>
  
 
	

<script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"pluginRootPath":"live2dw/","pluginJsPath":"lib/","pluginModelPath":"assets/","model":{"jsonPath":"live2d-widget-model-wanko"},"display":{"position":"right","width":150,"height":300},"mobile":{"show":true},"log":false,"tagMode":false});</script></body>
</html>
